# -*- coding: utf-8 -*-
from scrapy_redis.spiders import RedisCrawlSpider
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import  Rule
import json
import datetime
from ArticleSpider.utils.common import get_md5
from ArticleSpider.items import LagouJobItemLoader,LagouJobItem
class LagouSpider(RedisCrawlSpider):
    #自定义setting文件
    custom_settings={
        'REDIS_HOST': 'redis',
        'REDIS_PORT': 6379,
        "DOWNLOAD_DELAY":3, #使用延迟下载
        #https://www.jianshu.com/p/df9c0d1e9087
        #开启随机延迟  http://www.zhongzhiheng.club/2018/03/10/%E5%8F%8D%E7%88%AC%E8%99%AB%E7%AD%96%E7%95%A5%E5%8F%8A%E5%BA%94%E5%AF%B9%E6%96%B9%E6%B3%95/
        "RANDOMIZE_DOWNLOAD_DELAY":True,
        "COOKIES_ENABLED":False,#关闭cookie
        'DEFAULT_REQUEST_HEADERS': {
            'Accept': 'application/json, text/javascript, */*; q=0.01',
            'Accept-Encoding': 'gzip, deflate, br',
            'Accept-Language': 'zh-CN,zh;q=0.8',
            'Connection': 'keep-alive',
            'Cookie': 'user_trace_token=20171015132411-12af3b52-3a51-466f-bfae-a98fc96b4f90; LGUID=20171015132412-13eaf40f-b169-11e7-960b-525400f775ce; SEARCH_ID=070e82cdbbc04cc8b97710c2c0159ce1; ab_test_random_num=0; X_HTTP_TOKEN=d1cf855aacf760c3965ee017e0d3eb96; showExpriedIndex=1; showExpriedCompanyHome=1; showExpriedMyPublish=1; hasDeliver=0; PRE_UTM=; PRE_HOST=www.baidu.com; PRE_SITE=https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3DsXIrWUxpNGLE2g_bKzlUCXPTRJMHxfCs6L20RqgCpUq%26wd%3D%26eqid%3Dee53adaf00026e940000000559e354cc; PRE_LAND=https%3A%2F%2Fwww.lagou.com%2F; index_location_city=%E5%85%A8%E5%9B%BD; TG-TRACK-CODE=index_hotjob; login=false; unick=""; _putrc=""; JSESSIONID=ABAAABAAAFCAAEG50060B788C4EED616EB9D1BF30380575; _gat=1; _ga=GA1.2.471681568.1508045060; LGSID=20171015203008-94e1afa5-b1a4-11e7-9788-525400f775ce; LGRID=20171015204552-c792b887-b1a6-11e7-9788-525400f775ce',
            'Host': 'www.lagou.com',
            'Origin': 'https://www.lagou.com',
            'Referer': 'https://www.lagou.com/',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
        }
    }
    name = 'lagou'
    redis_key = 'lagou:start_urls'
    allowed_domains = ['www.lagou.com']
    start_urls = ['https://www.lagou.com/']
    with open("lagou_cookie.json", 'r', encoding='utf-8') as json_file:
        cookie = json.load(json_file)
    # 当页面符合整个规则的时候就调用回调函数进行抓取
    rules = (
        Rule(LinkExtractor(allow=('zhaopin/.*')),follow=True),
        Rule(LinkExtractor(allow=('gongsi/\d+.html')),follow=True),
        #抓取具体的职位页面
        #allow符合规则的url，callback页面回掉处理，follow跟踪页面中url
        Rule(LinkExtractor(allow=r'jobs/\d+.html'), callback='parse_job', follow=True),

    )
    #解析拉钩网的职位
    def parse_job(self, response):
        crawl_time=	datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        crawl_update_time=crawl_time
        salary=response.css(".salary::text").extract()[0]
        salary=salary.split('-')
        min_salary=(salary[0].strip('k') if salary[0] else 0)
        max_salary=(salary[1].strip('k ') if salary[1] else 0)
        tags=response.css(".position-label li::text").extract()
        tags=(",".join(tags) if tags else '')

        item_loader = LagouJobItemLoader(item=LagouJobItem(), response=response)
        item_loader.add_css("title", ".job-name span::text")
        item_loader.add_value("crawl_time", crawl_time)
        item_loader.add_value("crawl_update_time", crawl_update_time)
        item_loader.add_value("url", response.url)
        item_loader.add_value("url_object_id", get_md5(response.url))
        item_loader.add_xpath("job_city", "//*[@class='job_request']/p/span[2]/text()")
        item_loader.add_xpath("work_years", "//*[@class='job_request']/p/span[3]/text()")
        item_loader.add_xpath("degree_need", "//*[@class='job_request']/p/span[4]/text()")
        item_loader.add_xpath("job_type", "//*[@class='job_request']/p/span[5]/text()")
        item_loader.add_value("min_salary",min_salary)
        item_loader.add_value("max_salary",max_salary)
        item_loader.add_css("publish_time", ".publish_time::text")
        item_loader.add_value("tags",tags)
        item_loader.add_css("job_advantage", ".job-advantage p::text")
        item_loader.add_css("job_desc", ".job_bt div")
        item_loader.add_css("job_addr", ".work_addr")

        item_loader.add_css("company_url", "#job_company dt a::attr(href)")
        item_loader.add_css("company_name", "#job_company dt a div h2::text")

        job_item = item_loader.load_item()

        return job_item
